
% comparison of hyps learners vs exact:
% check how well each does over a range of hyperparameters
% uses cvwl

addpath('gpml-matlab-v3.1-2010-09-27');
startup;
warning('off','all'); % gotta love playing with fire
n_expts = 1;
n_reps = 2;
n = 1000;
m = 50;

output = 'expt6_results';

results = zeros(n_expts, 10); % frac right, error, time for each method; and then mean square error of hyps
data = zeros(n_reps,3); % how well method 1 does each time; how well method 2 does
for expt = 1:n_expts
  fprintf('\nExpt: %i\n', expt);
  for trial = 1:n_reps
    fprintf('Trial: %i\n',trial);
    gendata(normrnd([-0.3; -0.6 - trial / 3.0; -0.4], 0.2), n);

    disp('CV with learning and approximation 1')
    tic;
    CV_L_LDA(1, m);
    t = toc; results(expt, 6) = results(expt, 6) + t;
    tmp = load('cv_results');
    data(trial, 2) = tmp(1);
    
    disp('CV with learning and approximation 0')
    tic;
    CV_L_LDA(0, m);
    t = toc; results(expt, 3) = results(expt, 3) + t;
    tmp = load('cv_results');
    data(trial, 1) = tmp(1);

    disp('CV')
    tic;
    cross_validate();
    t = toc; results(expt, 9) = results(expt, 9) + t;
    tmp = load('cv_results');
    data(trial, 3) = tmp(1);

  end
  av = sum(data(:,1)) / n_reps;
  results(expt,1) = av;
  vars = (data(:,1) - av) .^ 2;
  results(expt,2) = sqrt(sum(vars)/n_reps) / sqrt(n_reps);
  
  av = sum(data(:,2)) / n_reps;
  results(expt,4) = av;
  vars = (data(:,2) - av) .^ 2;
  results(expt,5) = sqrt(sum(vars)/n_reps) / sqrt(n_reps);
  
  av = sum(data(:,3)) / n_reps;
  results(expt,7) = av;
  vars = (data(:,3) - av) .^ 2;
  results(expt,8) = sqrt(sum(vars)/n_reps) / sqrt(n_reps);
  
%  tmp = load('learnt_hyps_error');
%  results(expt, 10) = tmp;
  
  %dlmwrite(output, results);
end
%error('because I say so')
results = results/n; % normalise
results(:,[3,6,9,10]) *= n; % not that part
results(:, 10) = n; % just as an FYI
dlmwrite(output, results);
% note that it's in row form: method 1 fraction correct, the variance thereof, time; sim. for method 2, 3; n
disp('Done');